from selenium import webdriver
from lxml import etree
import re
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC


class LagouSpider(object):
    driver_path = r'C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe'
    def __init__(self):
        self.driver = webdriver.Chrome(executable_path=LagouSpider.driver_path)
        self.url = "https://www.lagou.com/jobs/list_python?fromSearch=true&labelWords=&suginput="
        self.positions = []
    def run(self):
        self.driver.get(self.url)

        while True:
            WebDriverWait(self.driver, timeout=10).until(
                EC.presence_of_element_located((By.XPATH, '//div[@class="pager_container"]/span[last()]'))
            )

            source = self.driver.page_source
            self.parse_list_page(source) #获取该页面 职位url的详情
            try:
                next_btn = self.driver.find_element_by_xpath("//div[@class='pager_container']/span[last()]")
                if "pager_next_disabled" in next_btn.get_attribute('class'):
                    break
                else:
                    next_btn.click()
            except:
                print(source)
            time.sleep(1)

    def parse_list_page(self,source):
        html = etree.HTML(source)
        links = html.xpath("//a[@class='position_link']/@href")
        for link in links:
            self.request_detail_page(link)
            time.sleep(2)
    def request_detail_page(self,url):

        self.driver.get(url)
        self.driver.execute_script("window.open('%s')" % url)
        self.driver.switch_to_window(self.driver.window_handles[1]) #打开新页面 driver也切换到新页面
        source = self.driver.page_source
        self.parse_detail_page(source)
        self.driver.close() #关闭新页面
        self.driver.switch_to_window(self.driver.window_handles[0]) #切换回主页面
    def parse_detail_page(self,source):
        html = etree.HTML(source)
        position_name = html.xpath("//h2[@class='name']/text()")[0]
        company_name = html.xpath("//em[@class='fl-cn']/text()")[0]
        job_requests = html.xpath("//dd[@class='job_request']//span")
        salary = job_requests[0].xpath(".//text()")[0].strip()
        city = job_requests[1].xpath(".//text()")[0].strip()
        city = re.sub(r"[\s/]","",city)
        work_year = job_requests[2].xpath(".//text()")[0].strip()
        work_year = re.sub(r"[\s/]", "", work_year)
        education = job_requests[3].xpath(".//text()")[0].strip()
        education = re.sub(r"[\s/]", "", education)
        dec = "".join(html.xpath("//dd[@class='job_bt']//text()")).strip()
        position = {
            'position_name':position_name,
            'company_name':company_name,
            'salary':salary,
            'city':city,
            'work_year':work_year,
            'education':education,
            'dec':dec,
        }
        self.positions.append(position)
        print(position)
        print("*"*50)

if __name__ == "__main__":
    spider = LagouSpider()
    spider.run()